{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a921093",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "!pip install -U \"jax[cpu]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e80b577",
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/google-deepmind/gemma.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be4050cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "snapshot_download(repo_id=\"google/gemma-2b-flax\", local_dir=\"/local/path/gemma-2b-flax\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be8907dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "VARIANT = '2b' # @param ['2b', '2b-it', '7b', '7b-it'] {type:\"string\"}\n",
    "\n",
    "\n",
    "ckpt_path = '/local/path/gemma-2b-flax/2b/'\n",
    "vocab_path = '/local/path/gemma-2b-flax/tokenizer.model'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd6a2b85",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load parameters\n",
    "from gemma import params as params_lib\n",
    "params = params_lib.load_and_format_params(ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6908204c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "vocab = spm.SentencePieceProcessor()\n",
    "vocab.Load(vocab_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "954b1e90",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We use the `transformer_lib.TransformerConfig.from_params` function to\n",
    "# automatically load the correct configuration from a checkpoint. Note that the\n",
    "# vocabulary size is smaller than the number of input embeddings due to unused\n",
    "# tokens in this release.\n",
    "\n",
    "from gemma import transformer as transformer_lib\n",
    "\n",
    "config_2b = transformer_lib.TransformerConfig.from_params(\n",
    "    params,\n",
    "    cache_size=30  # Number of time steps in the transformer's cache\n",
    ")\n",
    "model_2b = transformer_lib.Transformer(config=config_2b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d45d365",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gemma import sampler as sampler_lib\n",
    "# Create a sampler with the right param shapes.\n",
    "sampler = sampler_lib.Sampler(\n",
    "    transformer=model_2b,\n",
    "    vocab=vocab,\n",
    "    params=params['transformer'],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ffb3ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_texts = [\"I love to\", \"Today is a\", \"What is the\"  ]\n",
    "\n",
    "out_data = sampler(\n",
    "    input_strings=prompt_texts,\n",
    "    total_generation_steps=6,  # number of steps performed when generating\n",
    "  )\n",
    "\n",
    "for input_string, out_string in zip(prompt_texts, out_data.text):\n",
    "  print(f\"Prompt:\\n{input_string}\\nOutput:\\n{out_string}\")\n",
    "  print()\n",
    "  print(10*'#')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b649f61",
   "metadata": {},
   "outputs": [],
   "source": [
    "import jax\n",
    "\n",
    "def get_attention_mask_and_positions(example: jax.Array,\n",
    "                                     pad_id : int,\n",
    "                                     )-> tuple[jax.Array, jax.Array]:\n",
    "    \"\"\"Builds the position and attention mask vectors from the given tokens.\"\"\"\n",
    "\n",
    "    pad_mask = example != pad_id\n",
    "\n",
    "    current_token_position = transformer_lib.build_positions_from_mask(pad_mask)\n",
    "    attention_mask = transformer_lib.make_causal_attn_mask(pad_mask)\n",
    "    return current_token_position, attention_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "647ea726",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import jax.numpy as jnp\n",
    "from gemma import transformer as transformer_lib\n",
    "import jsonlines\n",
    "\n",
    "params = params_lib.load_and_format_params(ckpt_path)\n",
    "\n",
    "output_path = \"golden_data_gemma-2b.jsonl\"  \n",
    "all_data_to_save = []\n",
    "\n",
    "for prompt_index in range(len(prompt_texts)):\n",
    "    prompt_text = prompt_texts[prompt_index]\n",
    "    one_sample_input = np.array([2]+vocab.encode(prompt_text))\n",
    "    expanded_one_sample_input = jnp.expand_dims(one_sample_input, axis=0)\n",
    "    pad_id = vocab.pad_id\n",
    "    get_attention_mask_and_positions(one_sample_input, pad_id)\n",
    "    # Build the position and attention mask vectors.\n",
    "    positions, attention_mask = get_attention_mask_and_positions(one_sample_input, pad_id)\n",
    "    print(f\"{expanded_one_sample_input=}, {positions=}, {attention_mask=}\")\n",
    "\n",
    "\n",
    "\n",
    "    # Foward pass on the input data.\n",
    "    # No attention cache is needed here.\n",
    "\n",
    "    logits, _ = model_2b.apply(\n",
    "    #     params,\n",
    "        {'params': params['transformer']},\n",
    "        expanded_one_sample_input,\n",
    "        positions,\n",
    "        None,              # Attention cache is None.\n",
    "        attention_mask,\n",
    "    )\n",
    "    print(f\"{logits=}\")\n",
    "    \n",
    "    # Prepare data to be saved  \n",
    "    data_to_save = {  \n",
    "        \"prompt\": prompt_texts[prompt_index],  \n",
    "        \"completion\": out_data.text[prompt_index],  \n",
    "        \"tokens\": [2]+vocab.encode(prompt_texts[prompt_index]),  \n",
    "        \"logits\": logits[0].tolist() #remove the batch dim and then tolist() for json serialization\n",
    "    }  \n",
    "    all_data_to_save.append(data_to_save)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53f4b01c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with jsonlines.open(output_path,'w') as f:    \n",
    "    f.write_all(all_data_to_save)\n",
    "\n",
    "\n",
    "\n",
    "print(f\"Data saved to {output_path}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
